import time

import pandas as pd
import requests
import tushare as ts
from lxml import html
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

from  emmongo import em_mongomanager as mongoManager

import datetime
from LSTM.variableset import EMVar
import pandas as pd
from sysdata import sys_base_data as sysBaseData
import traceback
from dateutil.parser import parse
import numpy as np

s = requests.Session()

retries = Retry(total=5,backoff_factor=0.1,status_forcelist=[ 500, 502, 503, 504])
s.mount('http://', HTTPAdapter(max_retries=retries))

def get_profit(code,format_result='df', sleep_sec=0 ,neet_insertDB=True):
    response = s.get('http://vip.stock.finance.sina.com.cn/corp/go.php/vISSUE_ShareBonus/stockid/'+code+'.phtml')
    if sleep_sec!=0:
        time.sleep(sleep_sec)
    if response.status_code == 200:
        # // table[ @ id = "sharebonus_1"] / tbody / tr / td / text()
        tree = html.fromstring(response.text.encode('gb2312','ignore'))
        trs = tree.xpath('//*[@id="sharebonus_1"]/tbody/tr')
        fh_l = []
        tmp = pd.DataFrame()
        for ele in trs:
            # 公告日期 送股(股)	转增(股)	派息(税前)(元) 除权除息日	股权登记日	红股上市日
            # 后面3个字段 是否是分红(配股) 配股的数量 配股的价格
            fields = ele.xpath('td/text()')
            if len(fields) == 7:
                d = {'code':code, 'publish_d':str(fields[0]), 'sg_count':str(fields[1]),
                     'zz_count':str(fields[2]), 'px_money':str(fields[3]),
                     'cqcx_d':str(fields[4]), 'gqdj_d':str(fields[5]), 'hgss_d':str(fields[6]),
                     'is_fenhong':True, 'pg_count':0, 'pg_price':0}
                fh_l.append(d)
                if neet_insertDB:
                    mongoManager.db[mongoManager.COL_PROFIT_DATA].update_one(
                        {'code': code, 'publish_d': str(fields[0])}, {'$set': d}, upsert=True)
                    print("%s 权息数据插入成功" % code)
                tmp = pd.DataFrame(fh_l)
        trs = tree.xpath('//*[@id="sharebonus_2"]/tbody/tr')
        tmp2 = pd.DataFrame()
        pg_l = []
        for ele in trs:
            fields = ele.xpath('td/text()')
            if len(fields) == 9:
                publish_d = str(fields[0])
                if not tmp[tmp['publish_d'] == publish_d].empty:
                    if tmp[tmp['publish_d'] == publish_d].shape[0] != 1:
                        raise Exception('DDD')
                    index = (tmp[tmp['publish_d'] == str(fields[0])].index)[0]
                    d = tmp.ix[index].copy()
                    d['cqcx_d'] = str(fields[4])
                    d['gqdj_d'] = str(fields[5])
                    d['hgss_d'] = str(fields[8])
                    d['pg_count'] = fields[1]
                    d['pg_price'] = fields[2]
                    tmp.ix[index] = d
                    print('****^^^^^**** %s 在 %s 日同时发布分红与配股的公告' % (code, publish_d))
                else:
                    # hgss_d 是指配股的上市日
                    d = {'code': code, 'publish_d': publish_d, 'sg_count': 0,
                         'zz_count': 0, 'px_money': 0,
                         'cqcx_d': str(fields[4]), 'gqdj_d': str(fields[5]), 'hgss_d': str(fields[8]),
                         'is_fenhong':False, 'pg_count': fields[1], 'pg_price': fields[2]}
                    pg_l.append(d)
                if neet_insertDB:
                    mongoManager.db[mongoManager.COL_PROFIT_DATA].update_one(
                        {'code': code, 'publish_d': str(fields[0])}, {'$set': d}, upsert=True)
                    print("%s 配股数据插入成功" % code)
                tmp2 = pd.DataFrame(pg_l)
        df = pd.merge(tmp, tmp2, how='outer').sort_values('publish_d')
        if 'df' == format_result:
            return df
        elif 'list' == format_result:
            return list(df.to_dict(orient='index').values())
    else:
        raise Exception('%s 获取分红数据失败:状态码:%d'%(code, response.status_code))

def get_profit_cache_or_web(code, neet_insertDB=False):
    # r = mongoManager.db[mongoManager.COL_PROFIT_DATA].find({'code':code}).sort('last_update_time',-1)
    # if r.count() > 0:
    #     l = list(r)
    #     if (datetime.datetime.now() - parse(l[0]['last_update_time']) ).days < 5:
    #         return pd.DataFrame(l), True
    return get_profit(code, neet_insertDB=neet_insertDB), False


def compare_fq_fator(code,sleep_sec=0, neet_insertDB=True):

    def prepare_df(df):
        date_series = pd.to_datetime(df[EMVar.date])
        df.set_index(date_series, inplace=True)

    if sleep_sec != 0:
        time.sleep(sleep_sec)

    sfund = sysBaseData.get_self_fund(code)
    ipo_date = sfund.ipodate.strftime(EMVar.FORMATE_YMD)
    end = datetime.datetime.now().strftime('%Y-%m-%d')

    sina_profit_df, is_local = get_profit_cache_or_web(code, neet_insertDB=False)
    if sina_profit_df.empty:
        print('%s 没有任何分配公告' % (code))
        return
    sina_profit_df = sina_profit_df.sort_values('publish_d')

    if not is_local:
        noau_ts_his_df = ts.get_k_data(code=code, index=False, autype=None, start=ipo_date, end=end)
        qfq_ts_his_df = ts.get_k_data(code=code, index=False, autype='qfq', start=ipo_date, end=end)
        hfq_ts_his_df = ts.get_k_data(code=code, index=False, autype='hfq', start=ipo_date, end=end)

        prepare_df(noau_ts_his_df)
        prepare_df(qfq_ts_his_df)
        prepare_df(hfq_ts_his_df)
        l_ = []
        for index, value in sina_profit_df.iterrows():

            cq_date = value['cqcx_d']
            if cq_date == '--':
                print('%s %s 公告不分配预案'%(code, value['publish_d']))
                continue

            # SINA 需要直接拿到除权除息日当天的数据，
            # 因为是直接拿当天的数据与原始的未复权的数据做匹配计算复权因子
            sina_non_cq_item = dict(noau_ts_his_df[:cq_date].ix[-1])
            hfq_cq_item = dict(hfq_ts_his_df[:cq_date].ix[-1])
            hfq_factor = hfq_cq_item[EMVar.close] / sina_non_cq_item[EMVar.close]
            value['sum_hfq_factor'] = hfq_factor

            # 前复权是要拿除权除息日前一天的价格做比值，刚好就是跟后复权相反方向
            # 为什么后复权不是拿除权日的后一天价格做比值? 因为除权日后一天跟除权日当天与 没除权 时候的比值的因子是相同的
            sina_near_non_cq_item = dict(noau_ts_his_df[:cq_date].ix[-2])
            qfq_cq_item = dict(qfq_ts_his_df[:cq_date].ix[-2])
            qfq_factor = qfq_cq_item[EMVar.close] / sina_near_non_cq_item[EMVar.close]
            value['sum_qfq_factor'] = qfq_factor

            # 直到除权除息日的前一天的数据,这里是手动用wind的方法计算复权因子，
            # 所以需要拿除权除息日的前一天的数据，用来计算复权后的昨收价
            non_cq_item = dict(noau_ts_his_df[:cq_date].ix[-2])
            value['neary_trade_date'] = non_cq_item[EMVar.date]
            value['last_update_time'] = EMVar.STR_NOW_YMD_HM

            # 除权除息当天的价格
            is_fenhong = value['is_fenhong']
            # 除权前一天的收盘价
            non_cq_item_close = non_cq_item[EMVar.close]
            zz_count = float(value['zz_count'])
            sg_count = float(value['sg_count'])
            px_money = float(value['px_money'])
            pg_pct = float(value['pg_count']) / 10
            pg_price = float(value['pg_price'])
            fq_item_close = (non_cq_item_close - px_money / 10 + pg_price * pg_pct) / (1 + sg_count / 10 + zz_count / 10 + pg_pct)
            fq_item_close = float('%.2f'%fq_item_close)
            value['manual_moment_qfq_factor'] = fq_item_close / non_cq_item_close
            value['manual_moment_hfq_factor'] = non_cq_item_close / fq_item_close
            value['non_cq_item_close'] = non_cq_item_close
            value['wind_fq_item_close'] = fq_item_close
            l_.append(dict(value))

        sina_profit_df = pd.DataFrame(l_)

        s_wind_sum_qfq_factor = pd.Series()
        s_manual_moment_qfq_factor = sina_profit_df['manual_moment_qfq_factor']
        for index, item in s_manual_moment_qfq_factor.items():
            tmp = s_manual_moment_qfq_factor[index:]
            s_wind_sum_qfq_factor.set_value(index, list(np.cumprod(tmp))[-1])

        sina_profit_df['wind_sum_qfq_factor'] = s_wind_sum_qfq_factor
        sina_profit_df['wind_sum_hfq_factor'] = np.cumprod(sina_profit_df['manual_moment_hfq_factor'])
        if neet_insertDB:
            for index, value in sina_profit_df.iterrows():
                cq_date = value['cqcx_d']
                mongoManager.db[mongoManager.COL_PROFIT_DATA].update_one({'code': code, 'cqcx_d': cq_date},
                                                                         {'$set': dict(value)}, upsert=True)
                print("%s %s 权息数据更新成功" % (code, cq_date))
        return sina_profit_df
    else:
        return sina_profit_df

def get_total_profit_from_sina():
    t1 = ts.get_stock_basics()
    l1 = []
    t1_dict = t1.to_dict(orient='index')
    for code, data in t1_dict.items():
        code_profit = get_profit(code=code, format_result='list', sleep_sec=0.5)
        l1.extend(code_profit)
    return pd.DataFrame(l1)

def get_total_profit_with_factor():
    t1 = ts.get_stock_basics()
    l1 = []
    t1_dict = t1.to_dict(orient='index')
    for code, data in t1_dict.items():
        try:
            compare_fq_fator(code, sleep_sec=3)
        except Exception as e:
            # print(e)
            print('%s 获取失败 **************************** \n\n'%code)
            traceback.print_tb(e.__traceback__)

            continue


def fq_code_real_time(code, start=None, end=None,autype='qfq'):

    sfund = sysBaseData.get_self_fund(code)
    start = start or sfund.ipodate.strftime(EMVar.FORMATE_YMD)
    end = end or EMVar.format_now()

    cal_his_df = fq_code(code, start, end, autype)
    # 下面是拿自己经过复权的 以及 tushare上复权的数据进行对比，看复权的结果是否正确
    if 'sinac_qfq_close' in list(cal_his_df.columns):
        qfq_ts_his_df = ts.get_k_data(code=code, index=False, autype='qfq', start=start, end=end)
        compare_his_df = pd.merge(cal_his_df, qfq_ts_his_df, left_on=EMVar.date, right_on=EMVar.date, how='left', suffixes=('_cal', '_qfq_sina'))
    if 'sinac_hfq_close' in list(cal_his_df.columns):
        hfq_ts_his_df = ts.get_k_data(code=code, index=False, autype='hfq', start=start, end=end)
        compare_his_df = pd.merge(cal_his_df, hfq_ts_his_df, left_on=EMVar.date, right_on=EMVar.date, how='left', suffixes=('_cal', '_hfq_sina'))
    pass


# 这里是将原始未复权的价格与这只股票历史的复权的信息做连接
# 连接的时候只取除权前一天的价格，当天的价格，除权后一天的价格
def fq_test_code(noau_ts_his_df, cq_factor_df, autype='qfq'):
    noau_ts_his_df['dp1_date'] = noau_ts_his_df[EMVar.date].shift(-1)
    noau_ts_his_df['dm1_date'] = noau_ts_his_df[EMVar.date].shift(1)

    l = []
    for index, item in cq_factor_df.iterrows():
        cqcx_d = item['cqcx_d']
        # 除权前一天的未复权的价格
        dm_item = noau_ts_his_df[noau_ts_his_df['dp1_date'] == cqcx_d].iloc[0]
        # 除权当天的未复权的价格
        cqday_item = noau_ts_his_df[noau_ts_his_df[EMVar.date] == cqcx_d].iloc[0]
        # 除权后一天的未复权的价格
        dp_item = noau_ts_his_df[noau_ts_his_df['dm1_date'] == cqcx_d].iloc[0]
        l.append(cqday_item)
        l.append(dm_item)
        l.append(dp_item)
    noau_ts_his_df = pd.DataFrame(l).sort_values(EMVar.date)
    if autype == 'qfq':
        # cqcx_d
        # 将上面用到的那些日期的项与复权因子的DF进行拼接
        # 因为这里的除权因子对应的是除权日的前一天，所以拼接的时候是要拿前一天作为2边的key
        merge_df = pd.merge(noau_ts_his_df, cq_factor_df, left_on=EMVar.date, right_on='neary_trade_date', suffixes=('', '_y'),
                            how='left')
    else:
        # cqcx_d  neary_trade_date
        # 将上面用到的那些日期的项与复权因子的DF进行拼接
        merge_df = pd.merge(noau_ts_his_df, cq_factor_df, left_on=EMVar.date, right_on='cqcx_d',suffixes=('', '_y'), how='left')
    return merge_df

# 复权某只股票的价格，默认是前复权
def fq_code(code, start, end, autype='qfq'):

    def prepare_df(df):
        date_series = pd.to_datetime(df[EMVar.date])
        df.set_index(date_series, inplace=True)

    noau_ts_his_df = ts.get_k_data(code=code, index=False, autype=None, start=start, end=end)

    if not autype or autype not in ['qfq', 'hfq']:
        prepare_df(noau_ts_his_df)
        return noau_ts_his_df
    else:
        df = compare_fq_fator(code)

        fq_columns = ['neary_trade_date', 'wind_sum_qfq_factor', 'wind_sum_hfq_factor', 'cqcx_d', 'sum_qfq_factor',
                      'sum_hfq_factor']
        # 下面主要是将复权日当天的复权因子 迁移到 整个时间段中
        if autype == 'qfq':
            # merge_df = pd.merge(noau_ts_his_df, cq_factor_df, left_on=EMVar.date, right_on='neary_trade_date', suffixes=('', '_y'), how='left')
            merge_df = fq_test_code(noau_ts_his_df, df, autype)
            merge_df = merge_df[list(noau_ts_his_df.columns) + fq_columns]
            # bfill backfill :用后面的非 nan 数据填充前面的nan数据
            merge_df = merge_df.fillna(method='bfill')
            # 将 sum_qfq_factor 剩下为 nan的数据置为1
            merge_df['sum_qfq_factor'] = merge_df['sum_qfq_factor'].fillna(1)
            merge_df['wind_sum_qfq_factor'] = merge_df['wind_sum_qfq_factor'].fillna(1)
        else:
            # merge_df = pd.merge(noau_ts_his_df, df, left_on=EMVar.date, right_on='cqcx_d', suffixes=('', '_y'),how = 'left')
            merge_df = fq_test_code(noau_ts_his_df, df, autype)
            merge_df = merge_df[list(noau_ts_his_df.columns) + fq_columns]
            # pad :用之前的 非 nan 的数据填充 后面的 nan数据
            merge_df = merge_df.fillna(method='pad')
            merge_df['sum_hfq_factor'] = merge_df['sum_hfq_factor'].fillna(1)
            merge_df['wind_sum_hfq_factor'] = merge_df['wind_sum_hfq_factor'].fillna(1)

        noau_ts_his_df = merge_df[list(noau_ts_his_df.columns) + fq_columns]
        if autype == 'qfq':
            noau_ts_his_df['sinac_qfq_close'] = noau_ts_his_df['close'] * noau_ts_his_df['sum_qfq_factor']
            noau_ts_his_df['wind_qfq_close'] = noau_ts_his_df['close'] * noau_ts_his_df['wind_sum_qfq_factor']
        else:
            noau_ts_his_df['sinac_hfq_close'] = noau_ts_his_df['close'] * noau_ts_his_df['sum_hfq_factor']
            noau_ts_his_df['wind_hfq_close'] = noau_ts_his_df['close'] * noau_ts_his_df['wind_sum_hfq_factor']
        prepare_df(noau_ts_his_df)
        return noau_ts_his_df


# get_profit('300444',neet_insertDB=True)
# get_total_profit_from_sina()

# compare_fq_fator('300444')
# get_total_profit_with_factor()

# compare_fq_fator('000538')
# , autype='hfq'
fq_code_real_time('000538', autype='qfq')
